#include "episodic_memory.h"
#include <cstdlib>
#include <iostream>
#include <fstream>
#include <map>
#include <utility>
#include <string>
#include <math.h>
#include <ctime>
using namespace std;


	typedef map<string,string> csi;
	typedef map<string,string> epmem;
	typedef map<vector<string>,pair<double,int> > topcsi; 

	epmem_cue_learning::epmem_cue_learning() {
		best_csi = csi(); //the best csi that is remembered
        curr_csi = csi(); //the 0.current csi that is being operated on
		example_epmem = epmem();
        best_cost = 0; //the step cost associated with the best_csi
        best_num_uses = 0; //the number of times we have evaluated best_csi
        curr_cost = 0; //the step cost associated with the curr_csi
		total_runs = 0;//total number of tiems we run
		ave_cost = 0;//the overall average
		top_ten = topcsi();//keep track of top ten previously seen csis
		
		TERMINAL_KEYWORD = "reward";
	}
	//bootstrapper, takes in the example memory, used for randomization early on
	void epmem_cue_learning::initialize(epmem_cue_list ex_mem)
	{
		for (epmem_cue_list::iterator cli = ex_mem.begin(); cli != ex_mem.end(); ++cli) {
			example_epmem[cli->second->name] = cli->second->value;
		}		//example_epmem = ex_mem;//an example epmem for randomness
	}



	/* trimCue
	 *	
	 *	trims the input and returns the trimmed csi	
	*/
	epmem_cue_list epmem_cue_learning::trimCue(agent *my_agent,epmem_cue_list cloowts) {
		//use the curr_csi to trim the cloots
		
		epmem_cue_list  trimmed_cue_list =  epmem_cue_list();
		epmem_cue_list::iterator it; //it is the iterator for the epmem
		//foreach item in the curr_csi
		if (curr_csi.size() == 0)
		{
			//if it is null, then we need to randomly initialize it
			//with things that exist in the epmem
			epmem_cue_list rand_list = randomizeCSI(cloowts);
			for(epmem_cue_list::iterator cli = rand_list.begin(); 
				cli != rand_list.end(); cli++)
			{
				curr_csi[cli->second->name]=cli->second->value;
			}
			//rand_list is the whole trimmed list: AMN wants the leafs
			//return a epmem_cue_list to extract leaves in his code
			return rand_list;
		}
		for(csi::iterator itr = curr_csi.begin(); itr != curr_csi.end(); ++itr) {
			//it = the result of finding the curr_csi item			
			string name = itr->first;
			it = findEpmemCueListIterator(cloowts, name);
				//cloowts.find(name); 
			if (it != cloowts.end()) {
				//since it is not the error, assign to trimmed
				
				trimmed_cue_list[it->first] = it->second;
			} else {
				//report error, the epmem does not contain the attribute
				ofstream error;
				error.open("tankERR.txt", ios::app);
				if (error.is_open())
				{
					error << "trimCue: the curr_csi attribute " << name << " does not exist in epmem" << endl;
				}
				error.close();
			}
		}
		return trimmed_cue_list;
	}

	csi epmem_cue_learning::trimCue(epmem cloowts, csi trim_csi) {
		//use the trim_csi to trim the cloots
		csi trimmed_csi;
		epmem::iterator it; //it is the iterator for the epmem
		//foreach item in the trim_csi
		if (trim_csi.size() == 0)
		{
			//if it is null, then we need to randomly initialize it
			//with things that exist in the epmem
			
			trim_csi = randomizeCSI(cloowts);
			return trim_csi;
		}
		for(csi::iterator itr = trim_csi.begin(); itr != trim_csi.end(); ++itr) {
			//it = the result of finding the trim_csi item			
			string attribute = itr->first;
			it = cloowts.find(attribute); 
			if (it != cloowts.end()) {
				//since it is not the error, assign to trimmed
				trimmed_csi[attribute] = it->second;
			} else {
				//report error, the epmem does not contain the attribute
				ofstream error;
				error.open("tankERR.txt", ios::app);
				if (error.is_open())
				{
					error << "trimCue: the trim_csi attribute " << attribute << " does not exist in epmem" << endl;
				}
				error.close();
			}
		}
		return trimmed_csi;
	}

	/* recieveStepCost
	 *	
	 *	recieves c and does work on that step cost		
	*/
	void epmem_cue_learning::recieveStepCost(agent * my_agent,int c) {
		//set the curr_cost to what we just recieved
		curr_cost = c;
		//use the curr_cost to update total average experienced so far
		ave_cost = ((ave_cost*total_runs) + curr_cost)/(total_runs + 1);
		++total_runs;
		//update all variables needed for learning
		updateTopTen();

		//begin learning
		//first, select N "random" csi's
		//first one is going to be the best csi
		vector<csi> learning_csi = vector<csi>();

		//push back best
		learning_csi.push_back(best_csi);		
		int random;
		topcsi temp_top_csi = top_ten;
		for (int i = 0; i < AMOUNT_OF_CSI_FROM_TOP && temp_top_csi.size() != 0; ++i) {
			//get random index
			random = rand() % (temp_top_csi.size());
			//use the copying to get the iterator
			topcsi::iterator itr = temp_top_csi.begin();
			//"random access" loop
			for (int j = 0; j < random; ++j, ++itr);
			//grab the vector representing our csi
			vector<string> vec_attr = itr->first;
			//new csi being created
			csi temp_csi = csi();
			for (vector<string>::iterator vit = vec_attr.begin(); vit != vec_attr.end(); ++vit) {
				//append that to a new csi, then add that csi to learning_csi vector
				temp_csi[(*vit)] = "";
			}
			learning_csi.push_back(temp_csi);
			temp_top_csi.erase(itr);
		}
		int stopping_size = learning_csi.size();
		for (int i = 0; i < NUM_OF_CSI - stopping_size; ++i) {
			//randomly choose between one of the following options
			// 1. MODIFY: choose one of the current csi's in learning_csi and modify
			// 2. RANDOM: add a completely random csi to learning_csi
			random = rand() % 100;
			if (random < 50) {
				//MODIFY
				random = rand() % learning_csi.size();
				learning_csi.push_back(modifyCSI(learning_csi[random]));
			} else {
				//RANDOM
				learning_csi.push_back(randomizeCSI(example_epmem));
			}
		}

		//now begin our learning on the list
		evaluateCSI(my_agent,learning_csi);
	}

	void epmem_cue_learning::evaluateCSI(agent * my_agent,vector<csi> learning_csi) {
		//evaluate each csi individually and determine the best one
		vector<double> learning_cost = vector<double>();
		for (int i = 0; i < learning_csi.size(); ++i) {
			//grab a random memory for algorithm below
			//keep the seed at same epmem from episodic store
			epmem seed_epmem = getEPMEM(my_agent); 
			epmem learn_epmem = seed_epmem;
			//grab id number from epmem
			int id = atoi(seed_epmem["id"].c_str());
			//while not a terminal state, keep stepping
			int seed_step_c = 0;
			while (seed_epmem.find(TERMINAL_KEYWORD) == seed_epmem.end()) {
				seed_epmem = getEPMEM(my_agent,++id);
				++seed_step_c;
			}

			//begin to use csi to trim the learn epmem and step
			int curr_step_c = 0;
			csi trimmed = csi();
			//do the same as above, get states til we get a terminal state.
			while (learn_epmem.find(TERMINAL_KEYWORD) == learn_epmem.end()){
				//trim the learn_epmem with csi
				trimmed = trimCue(learn_epmem, learning_csi.at(i));				
				//get memory retrieval
				learn_epmem = getEPMEM(trimmed);
				//increment our step cost
				++curr_step_c;
				//get next in temporal order
				int id = atoi(learn_epmem["id"].c_str());
				learn_epmem = getEPMEM(my_agent,++id);
			}
			//populate the vector of costs YAY
			learning_cost.push_back(abs(seed_step_c - curr_step_c));			
		}
		//evaluate the costs
		//find the max
		int max_cost = 0;
		for (int i = 0; i < learning_cost.size(); ++i)
		{
			if (learning_cost.at(i) > max_cost) {
				max_cost = learning_cost.at(i);
			}
		}
		//now do maths
		int total_amount = 0;
		for (int i = 0; i < learning_cost.size(); ++i)
		{
			learning_cost[i] = learning_cost.at(i) / max_cost;
			if (learning_cost.at(i) != 0) {
				learning_cost[i] = 1 / learning_cost.at(i);
			}
			else {
				learning_cost[i] = 1000000;
			}
			total_amount += learning_cost.at(i);
		}
		
		srand(time(new time_t()));
	
		int random = rand()%total_amount;
		int idx_to_use = 0;
		//ok we have our random, now find where it goes.
		for (int i = 0; i < learning_cost.size(); ++i){
			if (random < learning_cost[i])
			{
				idx_to_use = i;
				break;
			}
			else
				random -= learning_cost[i];
		}
		//omg last step
		curr_csi = learning_csi[idx_to_use];
	}


	void epmem_cue_learning::updateTopTen()
	{
		//DESCRIPTION:
		//check to see if the curr_csi is in top ten
		//if so: update it and check if it is the best
		//if not: check to see if it should be
			//if so: remove worst top ten csi

		//iterate through each topcsi
		bool curr_csi_in_top_ten = false;
		double best_score = 1000000;
		//assuming cant get negative scores
		double worst_score = -10000000;
		topcsi::iterator best_itr;
		topcsi::iterator worst_itr;
		topcsi::iterator match_itr;

		for(topcsi::iterator itr = top_ten.begin(); itr != top_ten.end(); ++itr) {		
			//check to see if the curr_csi is the same size as itr vector
			vector<string> vec_attr = itr->first;
			if (curr_csi.size() == vec_attr.size()) {
				//now check to see if everything in the vector exists in the curr_csi
				int counter = 0;
				for (vector<string>::iterator vit = vec_attr.begin(); vit != vec_attr.end(); ++vit) {
					if (curr_csi.find(*vit) != curr_csi.end()) {
						//they match
						++counter;
					}//if
					else {
						//if one doesn't match, break out of searching vector
						break;
					}
				}//inner for
				if (counter == vec_attr.size()) {
					//they are the same size and have same elements
					//update the internals (itr->second.first = cost) (itr->second.second = num runs)
					itr->second.first = ((itr->second.first*itr->second.second) + curr_cost)/(itr->second.second + 1);
					++itr->second.second;
					curr_csi_in_top_ten = true;
					match_itr = itr;
				}//if a perfect match
			}//if same size			

			//update the best and worse memory
			if ((itr->second).first > worst_score) {
				worst_score = (itr->second).first;
				worst_itr = itr;
			} //if worse
			if ((itr->second).first < best_score) {
				best_score = (itr->second).first;
				best_itr = itr;
			}
		}//outer for

		//test remaining conditions
		if (!curr_csi_in_top_ten) {
			//check to see if it should be
			//if worst score is worse then the current, put current in
			//if worst is still initialized to -1, nothing is in
			if (worst_score > curr_cost || worst_score == -1
				|| top_ten.size() < MAX_TOP_LIST_SIZE) {
				//remove the worst, add the curr to top_ten	
				if (top_ten.size() == MAX_TOP_LIST_SIZE) {
					top_ten.erase(worst_itr);
				}
				vector<string> insert_vec;
				for (csi::iterator itr = curr_csi.begin(); itr != curr_csi.end(); ++itr) {
					insert_vec.push_back((itr->first));
				} // for
				pair<double, int> insert_pair(curr_cost, 1);
				top_ten[insert_vec] = insert_pair;
			}// if should be
			if (best_score > curr_cost){
				//best csi is now equal to current csi
				best_csi = curr_csi;
				best_cost = curr_cost;
				best_num_uses = 1;
			}
		} // if not in top ten
		else {
			// update the info about best if this new current one, is the best
			if (best_itr == match_itr) {
				csi new_best;
				for each (string s in (match_itr->first)) {
					new_best[s] = "";
				}
				//the best csi now knows its best cost
				best_csi = new_best;
				best_cost = (match_itr->second).first;
				best_num_uses = (match_itr->second).second;
			}
		}
	}
	
	epmem epmem_cue_learning::getEPMEM(agent *my_agent) {
		//for now
		return epmem_by_random_id(my_agent);
	}

	epmem epmem_cue_learning::getEPMEM(agent *my_agent,int id) {
		
		return epmem_by_id(my_agent,id);
	}

	/* getEPMEM
	 *	
	 *	gets a new memory based on a csi
	 //%%%TODO: not yet implemented, take out the call to this, or write this
	*/
	epmem epmem_cue_learning::getEPMEM(csi set) {
		//for now
		return example_epmem;// asd

	}

	// randomizeCSI
	// Authors: JRF_MJC
	// RandomizeCSI takes an epmem_cue_list for when we are passed an
	// epmem_cue_list by the EPMEM agent and asked to trim even though
	// we do not have a curr_csi
	epmem_cue_list epmem_cue_learning::randomizeCSI(epmem_cue_list em) {		
		epmem_cue_list temp_cue_list = epmem_cue_list();
		epmem_cue_list temp_epmem = em;
		int bad_idxs[SIZE_OF_CSI];
		for (int i = 0; i < SIZE_OF_CSI; ++i) {
			epmem_cue_list::iterator itr = temp_epmem.begin();
			//add to itr a random amount to simulate random access
			//i.e. simulates the += operator that some idiot never overrode
			//NOTE: may not be random		
			int rand_idx = rand() % temp_epmem.size();
			for (int j = 0; j < i;){
				if (rand_idx == bad_idxs[j]){
					//it is bad, grab a new one
					rand_idx = rand() % temp_epmem.size();
					j = 0;
				}
				else
					++j;
			}
			for (int j = 0; j < rand_idx; ++j, ++itr);				
			//temp_csi[itr->second->name] = itr->second->attribute; 
			temp_cue_list[itr->first] = itr->second;			
			bad_idxs[i] = rand_idx;
			//epmem_remove_entry_from_cuelist(&temp_epmem,itr);
		}
		return temp_cue_list;
	}

	// randomizeCSI
	// Authors: JRF_MJC
	// RandomizeCSI takes an epmem for when we do not have a curr_csi
	// or want to create a random one for our learning algorithm.
	csi epmem_cue_learning::randomizeCSI(epmem em) {		
		csi temp_csi = csi();
		epmem temp_epmem = em;
		for (int i = 0; i < SIZE_OF_CSI; ++i) {
			epmem::iterator itr = temp_epmem.begin();
			//add to itr a random amount to simulate random access
			//i.e. simulates the += operator that some idiot never overrode
			//NOTE: may not be random				
			int rand_idx = rand() % temp_epmem.size();
			for (int j = 0; j < rand_idx; ++j, ++itr);				
			temp_csi[itr->first] = itr->second; 
			temp_epmem.erase(itr);
		}
		return temp_csi;
	}

	
	csi epmem_cue_learning::modifyCSI(csi mod_me) {
		//for now, use example_epmem to randomly replace an attribute
		//future uses? : (look below) 
		// 1. One more trait added on
		// 2. One trait removed
		// 3. Don't use just example_epmem but store other epmems too
		epmem::iterator itr = example_epmem.begin();
		//add to itr a random amount to simulate random access
		//i.e. simulates the += operator			
		int rand_idx = rand() % example_epmem.size();
		for (int i = 0; i < rand_idx; ++i, ++itr);	

		//now grab random one to replace with
		rand_idx = rand() % SIZE_OF_CSI;
		csi::iterator it = mod_me.begin();
		for(int i = 0; i < rand_idx; ++i, ++it);

		//do the switch
		mod_me.erase(it);
		mod_me[itr->first] = "";

		return mod_me;
	}

	/****
	*Matthew Clark and Jordan Fryer
	*Linear search for a specific name in the cue list
	*@param cloowts the list of cues
	*@param name the string to match on in the second part of cloowts
	***/
	epmem_cue_list::iterator epmem_cue_learning::findEpmemCueListIterator(epmem_cue_list cloowts, string name)
	{
		for (epmem_cue_list::iterator itr = cloowts.begin(); itr != cloowts.end(); ++itr)
		{
			if ((itr->second->name) == name)
				return itr;
		}
		return cloowts.end();
	}

	
